﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using MIL.Html;
using System.Net;
using System.IO;
using Common;
using Dal;
using System.Text.RegularExpressions;

namespace Site_Extractor
{	
	public class SiteExtractor
	{

        public static bool IsContent(Uri Uri, List<string> IdentifyingUris)
        {
            if (IdentifyingUris.Count == 0)
                return true;
            foreach (string uri in IdentifyingUris)
            {
                if (Uri.ToString().ToLower().Contains(uri.ToLower()))
                    return true;
            }
            return false;
                  
        }

        public static bool IsStory(Story story)
        {
            if (story == null)
                return false;
            return (!(story.DateTime == DateTime.MinValue) && !(story.Title == string.Empty &&
                story.Text == string.Empty));
                
        }


		public static Story ExtractStory(string siteName, RuleSet ruleSet, Uri uri, string HtmlPage)
		{
            Story story = new Story();
            if (HtmlPage.Equals(string.Empty))
                return null;
            try
            {
                if (!IsContent(uri, ruleSet.IdentifyingUris)) //Not a Story.
                    return null;
                string page = HtmlPage;
                var nodes = HtmlDownloader.GetNodes(page);
                story.Title = HtmlAgilityPack.HtmlStripper.GetText(GetTextByRule(nodes, ruleSet[Rule.Title]));
                story.Summary = HtmlAgilityPack.HtmlStripper.GetText(GetTextByRule(nodes, ruleSet[Rule.Summary]));
                story.Text = HtmlAgilityPack.HtmlStripper.GetText(GetTextByRule(nodes, ruleSet[Rule.Story]));
                string sRawDateTime = GetTextByRule(nodes, ruleSet[Rule.Datetime]);
                //sRawDateTime = sRawDateTime.Replace(" ", "");
                story.DateTime = GetDateTimeFromString(sRawDateTime);
                story.SiteName = siteName;
                story.Uri = uri;

            }
            catch (Exception e)
            {
             
                Console.WriteLine(uri.ToString() + " Error creating story . Exception: {0}", e.Message);
                return null;
            }
			return story;
		}


        public static Story GetStory(Site site, Uri uri, string HTMLPage)
        {
            Uri u = uri;
            Story s = new Story();

            var page = HTMLPage;
            var text = HtmlStripper.GetTextFromHtml(page, true);
            var title = HtmlStripper.GetTitleFromHtml(page);
            var date = HtmlStripper.GetDateTimeFromHtml(page);

            s.Text = text;
            s.Title = title;
            s.DateTime = date;
            s.SiteName = site.SiteNames.SiteName;
            s.Summary = string.Empty;

            return s;
        }
        
        public static DateTime GetDateTimeFromString(string sRawDateTime)
        {
            string sDate = string.Empty;
            string sTime = string.Empty;
            Regex DateRegEx = new Regex(@"(\d+)[\\/.](\d+)[\\/.](\d+)");
            Regex TimeRegEx = new Regex(@"(\d+)[:](\d+)[^.^:]");
            Match DateMatch = DateRegEx.Match(sRawDateTime);
            Match TimeMatch = TimeRegEx.Match(sRawDateTime);

            int year, month, day, hours, minutes;
            if (!DateMatch.Success)
            {
                //Try Verbal Hebrew date.
                DateRegEx = new Regex(@"(\d+) ב(\w+) (\d+)");
                DateMatch = DateRegEx.Match(sRawDateTime);
                if (!DateMatch.Success)
                return DateTime.MinValue;
                else
                {
                    year = int.Parse(DateMatch.Groups[3].Value);
                    year = CorrectYear(year);
                    string monthString = string.Empty;
                    monthString = DateMatch.Groups[2].Value;
                    month = System.Globalization.DateTimeFormatInfo.CurrentInfo.MonthNames.ToList().IndexOf(monthString) + 1;
                    day = int.Parse(DateMatch.Groups[1].Value);
                }
                
            }
            else 
            {
                year = int.Parse(DateMatch.Groups[3].Value);
                year = CorrectYear(year);
                month = int.Parse(DateMatch.Groups[2].Value);
                day = int.Parse(DateMatch.Groups[1].Value);
            }
            if (!TimeMatch.Success)
            {
                //return DateTime.MinValue;
				hours = 0;
				minutes = 0;
            }
            else
            {
                hours = int.Parse(TimeMatch.Groups[1].Value);
                minutes = int.Parse(TimeMatch.Groups[2].Value);
            }
            DateTime dateTime = new DateTime(year, month, day, hours, minutes, 0);
            return dateTime;

        }

        private static  int CorrectYear(int year)
        {
            if (year < 2000)
                return year += 2000;
            else
                return year;
        }


		/// <summary>
		/// gathers all the text recursively from the nodes using the rule
		/// </summary>
		/// <param name="nodes"></param>
		/// <param name="rule"></param>
		/// <returns></returns>
		public static string GetTextByRule(HtmlNodeCollection nodes, Rule rule)
		{
			string text = string.Empty;
			//Get the node of the rule.
			var elements = GetNodesByRule(nodes, rule, rule.PartialValueMatch);
			//Rule is invalid, no such node found.
			if (null == elements) return text;
			//If there are no sub-rules, get all text below this node.
			if (rule.Rules.Count == 0)
			{
				string tmpTxt = string.Empty;
                foreach (HtmlElement e in elements)
                {
                    //
                    tmpTxt += GetTextRecursive(e.Nodes, rule.Exclusions); 
                    if (tmpTxt == string.Empty) tmpTxt += e.Text;
                }//e.Text; }
				return tmpTxt;
			}
			else
				foreach (HtmlElement e in elements)
				{
					foreach (Rule r in rule.Rules)
					{						
						text += GetTextByRule(e.Nodes, r);
					}
				}
			return text;
		}

        /// <summary>
        /// Gets Node described by the rule.
        /// </summary>
        /// <param name="nodes">some HTML node</param>
        /// <param name="rule">the rule describing the node</param>
        /// <returns>HTML node.</returns>
        public static HtmlNode GetNodeByRule(HtmlNodeCollection nodes, Rule rule)
        {
			foreach (HtmlNode childNode in nodes)
            {
                if (childNode is HtmlElement)
                {
                    HtmlElement e = childNode as HtmlElement;
                    if (e.Name.ToLower() == rule.PropertyName.ToLower() )
                    if( HasAttribValue(e.Attributes,rule.AttributeName,rule.AttributeValue))
                        return childNode;
                    HtmlNode ret = GetNodeByRule(e.Nodes, rule);
                    if (null != ret) return ret;
                }
            }
            return null;
        }

		/// <summary>
		/// gathers all the nodes within the collection that matches the rule
		/// </summary>
		/// <param name="nodes">collection within to search</param>
		/// <param name="rule">rule to match</param>
        /// <param name="partial">partial match for attribute value?</param>
		/// <returns>array of HtmlNode with results or null if none found</returns>
		private static HtmlNode[] GetNodesByRule(HtmlNodeCollection nodes, Rule rule, bool partial)
		{
			List<HtmlNode> found = new List<HtmlNode>();
			foreach (HtmlNode childNode in nodes)
			{
				if (childNode is HtmlElement)
				{
					HtmlElement e = childNode as HtmlElement;
                   if (IsElementRule(e, rule, partial) && !IsElementInExclusionList(e, rule.Exclusions))
                       found.Add(e);
                    
                    var ret = GetNodesByRule(e.Nodes, rule, partial);
					if (ret != null) foreach (HtmlNode n in ret) found.Add(n);
				}

			}
			if (found.Count != 0) return found.ToArray();
			return null;
		}

        /// <summary>
        /// Get all text under this node.
        /// </summary>
        /// <param name="nodes">some HTML node</param>
        /// <returns>text</returns>
        private static string GetTextRecursive(HtmlNodeCollection nodes, List<Rule> Exclusions)
        {
			string tmpString;
            string text = string.Empty;

            foreach (HtmlNode childNode in nodes)
            {
                
                if (childNode is HtmlText)
                {
					if (childNode.Parent.Name.ToLower() != "script" && childNode.Parent.Name.ToLower() != "img")
					{
						
						tmpString = ((HtmlText)childNode).Text;
						if (tmpString.Length == 0) continue;
						if (text.Length > 0 && text[text.Length - 1] != ' '  && tmpString[0] != ' ') text += ' ';
						text += tmpString;
					}
                }
                if (childNode is HtmlElement)
                {
					if (!IsElementInExclusionList(childNode as HtmlElement, Exclusions))
					//text += GetTextRecursive(((HtmlElement)childNode).Nodes, Exclusions);
					{
						tmpString = GetTextRecursive(((HtmlElement)childNode).Nodes, Exclusions);
						if (tmpString.Length == 0) continue;
						if (text.Length > 0 && text[text.Length - 1] != ' ' && tmpString[0] != ' ') text += ' ';
						text += tmpString;
					}
                }
            }
            return text;

        }

       /// <summary>
       /// Checks if Element Matches Rule
       /// </summary>
       /// <param name="element"></param>
       /// <param name="rule"></param>
       /// <param name="partial">partial match for all attributes?</param>
       /// <returns></returns>
        private static bool IsElementRule(HtmlElement element, Rule rule, bool partial)
        {
            if (element.Name.ToLower() == rule.PropertyName.ToLower()) //This is the property we're looking for.
            {
                bool hasAllAttributes = true;
                foreach (Common.Attribute a in rule.Attributes) //Check if all attributes in rule are accounted for.
                {
                    if (!HasAttribValue(element.Attributes, a.Name, a.Value, partial))
                        hasAllAttributes = false;
                }
                if (hasAllAttributes && HasAttribValue(element.Attributes, rule.AttributeName, rule.AttributeValue, partial))
                    return true;
            }
            return false;
        }

        private static bool IsElementInExclusionList(HtmlElement element, List<Rule> Exclusions)
        {
            bool b = false;
            foreach (Rule rule in Exclusions)
                if (IsElementRule(element, rule, false))
                b = true;
            return b;

        }

       /// <summary>
        /// checks whether attribute exists in the given collection. (by name and value).
        /// If attName and AttValue are empty strings, return TRUE only if collection is EMPTY.
       /// </summary>
       /// <param name="attribs"></param>
       /// <param name="attName"></param>
       /// <param name="attValue"></param>
       /// <param name="partial">Partial value match?</param>
       /// <returns></returns>
        private static bool HasAttribValue(HtmlAttributeCollection attribs, string attName, string attValue, bool partial)
        {

            if ((attName == Common.Attribute.WILDCARD) || (attName == string.Empty && attValue == string.Empty && attribs.Count == 0)) return true;

            foreach (HtmlAttribute att in attribs)
            {
                if (att.Name.ToLower() == attName.ToLower())
                {
                    if (attValue == Common.Attribute.WILDCARD) return true;
                    if (!partial)
                    {
                        try
                        {
                            if (att.Value.ToLower() == attValue.ToLower()) return true;
                        }
                        catch (Exception e)
                        {
                        }
                    }
                    else
                        if (att.Value.ToLower().Contains(attValue.ToLower())) return true;
                        
                }
               
            }
            return false;
        }

        private static bool HasAttribValue(HtmlAttributeCollection attribs, string attName, string attValue)
        {
            return HasAttribValue(attribs, attName, attValue, false);
        }

		

		private static string GetByDepth(MIL.Html.HtmlElement node, ExtractionDepth depth)
		{
			string text=string.Empty;
			switch (depth)
			{
				case ExtractionDepth.None:
					return node.Text;
				case ExtractionDepth.One:					
					foreach (HtmlNode childNode in node.Nodes)
					{
						if (childNode.IsText()) { text += ((HtmlText)childNode).Text + " "; }
						else
						{
							if (!(childNode is MIL.Html.HtmlElement)) continue;
							text += ((MIL.Html.HtmlElement)childNode).Text + " ";
						}

					}
					return text;
				case ExtractionDepth.Two:					
					var nds = ((MIL.Html.HtmlElement)node.Nodes[0]);
					foreach (HtmlNode childNode in nds.Nodes)
					{
						if (childNode.IsText()) { text += ((HtmlText)childNode).Text + " "; }
						else
						{
							if (!(childNode is MIL.Html.HtmlElement)) continue;
							text += ((MIL.Html.HtmlElement)childNode).Text + " ";
						}

					}
					return text;			
			}
			return null;
		}

       
      
	}
	
	
	
	
	
}
